import Core.WindFunctions as WindFunction
import Core.Gadget as Gadget
import Core.IO as IO
import Core.Global as Global
import Core.Config as Config
import DataProcess
import DataProcess.Download_Stock_Fundamental as Fundamental
from datetime import *
import decimal
import os

# 对库中报表的日期分布进行统计
def Statistics(database):
    total = 0
    completeData = 0
    period1 = 0
    period2 = 0
    period3 = 0
    period4 = 0
    instruments = database.findWithFilter("Instruments","Stock")
    for instrument in instruments:
        symbol = instrument["Symbol"]
        statements = database.findWithFilter("Fundamental",symbol + "_Fundamental")
        total = total + len(statements)
        for statement  in statements:
            if statement.get("Wind") != None and len(statement["Wind"]) >= 151:
                completeData = completeData + 1
                if statement["Period"] == 1:
                    period1 = period1 + 1
                if statement["Period"] == 2:
                    period2 = period2 + 1
                if statement["Period"] == 3:
                    period3 = period3 + 1
                if statement["Period"] == 4:
                    period4 = period4 + 1

    print("Total Financial Statements:" + str(total))
    print("Total Complete Data:" + str(completeData))
    print("Total Period1:" + str(period1))
    print("Total Period2:" + str(period2))
    print("Total Period3:" + str(period3))
    print("Total Period4:" + str(period4))


# ---历年来某一日发布日的情况---
def Statistics_Release_Progress_at_Specific_Day_in_Hist(database, month, day, period=4):
    #
    reports = database.Find("Stock", "Fundamental_Basic", filter={"Period": period}, projection=["symbol", "report_date", "release_date", "period"])
    #
    statistics = {}
    for report in reports:
        year = report["report_date"].year
        if period == 4:
            targetYear = year + 1
        else:
            targetYear = year
        #
        if statistics.get(targetYear) == None:
            item = {}
            item["Pre"] = 0
            item["Post"] = 0
            statistics[targetYear] = item
        #
        item = statistics.get(targetYear)
        dt = datetime(targetYear, month, day)
        dt = dt.date()
        reportDate = report["release_date"]
        if reportDate < dt:
            item["Pre"] += 1
        else:
            item["Post"] += 1
    #
    for k,v in statistics.items():
        totalCount = v["Pre"] + v["Post"]
        print("Year: ", str(k), "Pre:", str(v["Pre"]), "Post:", str(v["Post"]), "Ratio", v["Pre"] / totalCount)


# ---历年来不同月份发布日的情况---
def Statistics_ReleaseDateCount():
    datetime1 = datetime(2005,1,1)
    datetime2 = datetime(2019,5,15)

    dts = Gadget.GenerateBeginDateofMonth(datetime1, datetime2)
    lastDateTime = datetime1
    for curDateTime in dts:
        filter = {}
        filter["ReleaseDate"] = {"$gte": lastDateTime, "$lt":curDateTime}
        statements = database.Find("Stock", "Fundamental", filter)
        count = len(statements)
        print(lastDateTime, curDateTime, count)
        lastDateTime = curDateTime


def CheckField(database,catagory = "Wind", field = "WorkingCapital"):
    count = 0
    instruments = database.findAll("Instruments","Stock")
    for instrument in instruments:
        symbol = instrument["Symbol"]

        if symbol == "002523.SZ":
            kkwood = 0

        fundamentals = database.findAll("Fundamental",symbol + "_Fundamental")

        if fundamentals.__len__() == 0:
            #print(symbol + " NoReports")
            continue
        count = count + len(fundamentals)
        for statement in fundamentals:
            reportDate = statement["ReportDate"]

            if catagory not in statement:
                continue
            if field not in statement[catagory]:
                print(symbol + " " + reportDate + "Field:" + field + "Not include")
                #reportDateTime = Gadget.ParseDateTime(reportDate)
                #CollectReport(database,symbol,reportDateTime)
                continue
            if statement[catagory][field] > 0:

                print(symbol + " " + reportDate + " Checked Field " )
                totalEquity = statement[catagory]["tot_equity"]
                cs = statement[catagory]["cap_stk"]
                ts = statement[catagory]["tsy_stk"]
                ps = statement[catagory]["other_equity_instruments_PRE"]
                reserve1 = statement[catagory]["cap_rsrv"]
                reserve2 = statement[catagory]["surplus_rsrv"]
                reserve3 = statement[catagory]["prov_nom_risks"]
                minority = statement[catagory]["minority_int"]
                retend = statement[catagory]["undistributed_profit"]
                comprehensive = statement[catagory]["other_compreh_inc_bs"]

                kkwood = 0

    print("Total Report " + str(count))


def CheckMissingWindValues(database,instruments):
    for instrument in instruments:
        symbol = instrument["Symbol"]
        print(symbol)

        contents = []
        contents = WindFunction.FundamentalFields()
        fundamentals = database.findAll("Fundamental",symbol + "_Fundamental")
        for report in fundamentals:
            reportDate = report["ReportDate"]
            missing = 0
            for field in contents:
                if field not in report["Wind"]:
                    # print("Missing Field " + symbol + " " + reportDate + " Field " + field)
                    missing = missing + 1
            if missing > 0:
                print("Missing Field " + symbol + " " + reportDate + " Count " + str(missing))


# ---Figure out possible missing report (of investments) at a specific datetime---
# 时间区间内，找到缺失的财务报表
def CheckMissingReport(database, instruments, datetime1, datetime2):

    targetDates = Gadget.GenerateReportDates(datetime1, datetime2)

    missingReportsBySymbol = {}
    for instrument in instruments:
        realDateTime1, realDateTime2 = DataProcess.DeterminRange(instrument, datetime1, datetime2)
        missingReports = CheckMissingReportbyInstrument(database, instrument, realDateTime1, realDateTime2)
        if len(missingReports) != 0:
            missingReportsBySymbol[instrument["Symbol"]] = missingReports

    return missingReports


# ---Figure out possible missing report (of single investment) at a specific datetime---
def CheckMissingReportbyInstrument(database, instrument, datetime1, datetime2):
    #
    targetDates = Gadget.GenerateReportDates(datetime1, datetime2)
    return CheckMissingReportbyInstrument2(database, instrument, targetDates)


#
def CheckMissingReportbyInstrument2(database, instrument, targetDates):
    #
    symbol = instrument["symbol"]
    filter = {}
    filter["symbol"] = instrument["symbol"]
    fundamentals = database.Find("Stock", "Fundamental", filter)
    #
    if fundamentals.__len__() == 0:
        print(symbol, "NoReports @ ReportDate", targetDates)
        #return []
    #
    reportByDate = {}
    for report in fundamentals:
        reportDate = report["report_date"]
        reportDate = reportDate.strftime('%Y-%m-%d')
        reportByDate[reportDate] = report
    #
    missingReports = []
    for reportDate in targetDates:
        target = reportDate.strftime('%Y-%m-%d')
        # targetDate = Gadget.ToUTCDateTime(reportDate).date()

        # ---Loop and find---
        found = False
        missingValues = False
        if target in reportByDate:
            found = True
            #
            # if "Wind" not in report:
            #     missingValues = True
            # elif len(report["Wind"]) < 10:
            #     missingValues = True

        #
        if found == False or missingValues:
            print(symbol + " Missing Historical Report " + target)
            missingReports.append(reportDate)
            # if target not in missingReports:
            # missingReports[target] = []
            # missingReports[target].append(symbol)

    return missingReports

# 查找缺失：前十大股东信息
def CheckMissingShareHolders(database, instruments, datetime1, datetime2):

    missingFreeFloat = []
    missingShareHolders = []
    count = 0
    for instrument in instruments:
        count +=1
        symbol = instrument["Symbol"]
        print(symbol,count)

        #if symbol != "000033.SZ":
        #    continue
        #    k = 1

        realDateTime1, realDateTime2 = DataProcess.DeterminRange(instrument, datetime1, datetime2)
        targetDates = Gadget.GenerateReportDates(realDateTime1, realDateTime1)
        #
        reports = database.findAll("Fundamental", symbol + "_Fundamental")
        for reportDate in targetDates:
            target = reportDate.strftime('%Y-%m-%d')
            found = False
            for report in reports:
                if report["ReportDate"][:10] == target:
                    if "FreeFloatShareHolders" not in report:
                        missingFreeFloat.append([symbol,reportDate])
                    if "ShareHolders" not in report:
                        missingShareHolders.append([symbol,reportDate])
                    found = True

                if found:
                    break

            if found == False:
                print(symbol, " Can't find ReportDate@", reportDate)

    #
    print("Missing FreeFloat ShareHolders")
    for i in missingFreeFloat:
        print(i)

    print("Missing ShareHolders")
    for i in missingShareHolders:
        print(i)

    return missingFreeFloat, missingShareHolders


#---2017-5-12---
#---Total Expense 包含所有（Dep && Interest Expense）---
#--- OperatingProfit1 = Total Rev - Total + Interest
#--- OperatingProfit2 (Wind opprofit) = Total Rev - Total Exp + Invest
def Fix_OpratingProfit1(database, instruments):
    count = 0
    for instrument in instruments:
        count = count + 1
        symbol = instrument["Symbol"]
        print(symbol + " count:" + str(count))
        collectionName = symbol + "_Fundamental"
        fundamentals = database.findAll("Fundamental",collectionName)
        if len(fundamentals) == 0:
            print(symbol + " NoReport")
            continue

        for statement in fundamentals:
            reportDate = statement["ReportDate"]
            totalRev = statement["Values"]["TotalRevenue"]
            totalExp = statement["Values"]["TotalExpense"]
            interest = statement["Values"]["InterestExpense"]
            if interest > 0:
                kkwood = 0

            dep = statement["Values"]["DepAmort"]
            operatingProfit1 = totalRev - totalExp + interest
            eitda = operatingProfit1 + dep
            target = {"ReportDate":reportDate}
            values = {}
            values["Values.OperatingProfit1"] = operatingProfit1
            values["Values.EBITDA"] = eitda
            database.update("Fundamental",collectionName,target,values)
            kkwood = 1


def Fix_FreeCashFlow(database, instruments):
    count = 0
    for instrument in instruments:
        count = count + 1
        symbol = instrument["Symbol"]
        print(symbol + " count:" + str(count))
        collectionName = symbol + "_Fundamental"
        fundamentals = database.findAll("Fundamental",collectionName)
        if len(fundamentals) == 0:
            print(symbol + " NoReport")
            continue

        for statement in fundamentals:
            reportDate = statement["ReportDate"]

            if "FreeCashFlow" not in statement["Values"]:
                print(symbol + " Fix FreeCashFlow @" + reportDate)
            #
            cf = statement["Values"]["OperatingCashFlow"]
            exp = statement["Values"]["CapitalExpenditure"]
            fcf = cf - exp

            target = {"ReportDate":reportDate}
            values = {}
            values["Values.FreeCashFlow"] = fcf
            database.update("Fundamental", collectionName, target, values)
            kkwood = 1


def MissingReportList():
    missingReports = []
    # missingReports.append({"Symbol": "000620.SZ", "ReportDate": datetime(2008, 9, 30)})
    # missingReports.append({"Symbol": "000748.SZ", "ReportDate": datetime(2016, 12, 31)})
    missingReports.append({"Symbol": "000748.SZ", "ReportDate": datetime(2017, 3, 31)})
    missingReports.append({"Symbol": "600005.SH", "ReportDate": datetime(2016, 12, 31)})
    missingReports.append({"Symbol": "600005.SH", "ReportDate": datetime(2017, 3, 31)})
    missingReports.append({"Symbol": "000522.SZ", "ReportDate": datetime(2013, 3, 31)})
    missingReports.append({"Symbol": "000562.SZ", "ReportDate": datetime(2014, 12, 31)})
    missingReports.append({"Symbol": "000602.SZ", "ReportDate": datetime(2013, 6, 30)})
    missingReports.append({"Symbol": "000618.SZ", "ReportDate": datetime(2005, 12, 31)})
    missingReports.append({"Symbol": "000699.SZ", "ReportDate": datetime(2007, 3, 31)})
    missingReports.append({"Symbol": "000763.SZ", "ReportDate": datetime(2005, 12, 31)})
    missingReports.append({"Symbol": "000765.SZ", "ReportDate": datetime(2005, 3, 31)})
    missingReports.append({"Symbol": "000765.SZ", "ReportDate": datetime(2005, 6, 30)})
    missingReports.append({"Symbol": "000817.SZ", "ReportDate": datetime(2005, 12, 31)})
    missingReports.append({"Symbol": "200041.SZ", "ReportDate": datetime(2009, 9, 30)})
    missingReports.append({"Symbol": "600002.SH", "ReportDate": datetime(2006, 3, 31)})
    missingReports.append({"Symbol": "600092.SH", "ReportDate": datetime(2005, 12, 31)})
    missingReports.append({"Symbol": "600092.SH", "ReportDate": datetime(2006, 3, 31)})
    missingReports.append({"Symbol": "600092.SH", "ReportDate": datetime(2006, 6, 30)})
    missingReports.append({"Symbol": "600092.SH", "ReportDate": datetime(2006, 9, 30)})
    missingReports.append({"Symbol": "600102.SH", "ReportDate": datetime(2011, 12, 31)})
    missingReports.append({"Symbol": "600205.SH", "ReportDate": datetime(2007, 3, 31)})
    missingReports.append({"Symbol": "600253.SH", "ReportDate": datetime(2013, 6, 30)})
    missingReports.append({"Symbol": "600263.SH", "ReportDate": datetime(2011, 12, 31)})
    missingReports.append({"Symbol": "600286.SH", "ReportDate": datetime(2006, 12, 31)})
    missingReports.append({"Symbol": "600286.SH", "ReportDate": datetime(2007, 3, 31)})
    missingReports.append({"Symbol": "600296.SH", "ReportDate": datetime(2007, 3, 31)})
    missingReports.append({"Symbol": "600553.SH", "ReportDate": datetime(2010, 12, 31)})
    missingReports.append({"Symbol": "600591.SH", "ReportDate": datetime(2009, 12, 31)})
    missingReports.append({"Symbol": "600607.SH", "ReportDate": datetime(2009, 12, 31)})
    missingReports.append({"Symbol": "600659.SH", "ReportDate": datetime(2005, 6, 30)})
    missingReports.append({"Symbol": "600659.SH", "ReportDate": datetime(2005, 9, 30)})
    missingReports.append({"Symbol": "600659.SH", "ReportDate": datetime(2005, 12, 31)})
    missingReports.append({"Symbol": "600772.SH", "ReportDate": datetime(2005, 12, 31)})
    missingReports.append({"Symbol": "600772.SH", "ReportDate": datetime(2006, 3, 31)})
    missingReports.append({"Symbol": "600772.SH", "ReportDate": datetime(2006, 6, 30)})
    missingReports.append({"Symbol": "600772.SH", "ReportDate": datetime(2006, 9, 30)})
    missingReports.append({"Symbol": "600786.SH", "ReportDate": datetime(2007, 12, 31)})
    missingReports.append({"Symbol": "600799.SH", "ReportDate": datetime(2005, 12, 31)})
    missingReports.append({"Symbol": "600842.SH", "ReportDate": datetime(2009, 12, 31)})
    missingReports.append({"Symbol": "600991.SH", "ReportDate": datetime(2011, 12, 31)})
    # missingReports.append({"Symbol": "T00018.SH", "ReportDate": datetime(2006, 9, 30)})

    return missingReports


def MissingReport20180930():
    missingReports = []
    missingReports.append({"Symbol": "002680.SZ", "ReportDate": datetime(2018, 9, 30)})
    missingReports.append({"Symbol": "601313.SH", "ReportDate": datetime(2018, 9, 30)})
    missingReports.append({"Symbol": "300694.SZ", "ReportDate": datetime(2018, 9, 30)})
    missingReports.append({"Symbol": "601162.SH", "ReportDate": datetime(2018, 9, 30)})
    missingReports.append({"Symbol": "002940.SZ", "ReportDate": datetime(2018, 9, 30)})
    missingReports.append({"Symbol": "002939.SZ", "ReportDate": datetime(2018, 9, 30)})
    return missingReports


def MissingShareHolders20180930():
    missingReports = []
    datetime2 = datetime(2018, 9, 30)
    '''
    missingReports.append({"Symbol": "	000004.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	000662.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	000978.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	002492.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	002673.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	002709.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	002769.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	002786.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	300022.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	300071.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	300093.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	300166.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	300180.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	300265.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	300486.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	600775.SH	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	600818.SH	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	600835.SH	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	601929.SH	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	603116.SH	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	300515.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	002808.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	300581.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	002862.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	601326.SH	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	300719.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	603106.SH	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	000096.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	000407.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	000540.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	000597.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	000600.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	000690.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	000725.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	000889.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	002165.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	002339.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	002349.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	002559.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	002586.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	300009.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	300026.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	300035.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	300075.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	300163.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	300307.SZ	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	603268.SH	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	603286.SH	", "ReportDate": datetime2})
    missingReports.append({"Symbol": "	603383.SH	", "ReportDate": datetime2})
    '''
    missingReports.append({"Symbol": "	300760.SZ	", "ReportDate": datetime2}) #没有更新成功

    for m in missingReports:
        m["Symbol"] = m["Symbol"].strip()
    return missingReports


# 下载缺失财务报表
def Download_MissingReport(database, missingReports):

    for missingReport in missingReports:
        symbol = missingReport["Symbol"]
        reportDate = missingReport["ReportDate"]
        print("Try to Download Missing Report", symbol, reportDate)
        Fundamental.Download_Fundamentals(database, symbol, reportDate)
        kkwood = 0


# 下载缺失财务报表
def Download_MissingReport_FromCSV(database, pathfileName="d:/MissingWindDataReport.csv"):
    header,table = IO.ReadCSVFile(pathfileName)
    for missing in table:
        symbol = missing[0]
        reportDate = Gadget.ParseDateTime(missing[1],"%Y/%m/%d")
        print("Processing:" + missing[0] + " " + missing[1])
        Fundamental.Download_Fundamentals(database, symbol, reportDate)


def Download_MissingShareHolders(database, missingReports):

    for missingReport in missingReports:
        symbol = missingReport["Symbol"]
        reportDate = missingReport["ReportDate"]
        print("Try to Download Missing ShareHolders", symbol, reportDate)
        Fundamental.Download_SharesHolder(database, symbol, reportDate)
        kkwood = 0
    pass


def Fix_Unset(database, instruments):
    for instrument in instruments:
        symbol = instrument["Symbol"]
        print(symbol)
        fundamentals = database.findAll("Fundamental",symbol + "_Fundamental")
        coll =  database.client["Fundamental"][symbol + "_Fundamental"]
        for fund in fundamentals:
            reportDate = fund["ReportDate"]
            if "Values" in fund:
                update = {}
                update["$unset"] = {}
                update["$unset"]["Values.FCF"] = ""
                update["$unset"]["Values.InvstCashFlow"] = ""
                update["$unset"]["Values.PriceEarning"] = ""
                update["$unset"]["Values.PriceSale"] = ""
                update["$unset"]["Values.EPS_ttm"] = ""
                update["$unset"]["Values.Recivable"] = ""
                coll.update_one({"ReportDate":reportDate}, update, False)


def Fix_MissingField(database, field, windField):

    instruments = database.findWithFilter("Instruments","Stock")
    count = 0
    start = False
    for instrument in instruments:
        count = count + 1
        symbol = instrument["Symbol"]

        if symbol == "600005.SH":
            start = True
        if start == False:
            continue


        print("Count " + symbol + " " + str(count))
        collectionName = symbol + "_Fundamental"
        fundamentals = database.findAll("Fundamental",collectionName)
        if len(fundamentals) == 0:
            #print(symbol + " NoReport")
            continue

        #

        for statement in fundamentals:
            if field not in statement["Values"]:
                #
                rptDateStr = statement["ReportDate"]
                rptDateStr = rptDateStr[:10]
                params = {}
                params["rptDate"] = rptDateStr
                #params["endDate"] = rptDateStr
                params["tradeDate"] = rptDateStr
                #params["currencyType"] = ""
                params["rptType"] = "1"
                #params["priceAdj"] = "U"
                #params["cycle"] = "D"
                #
                values = WindFunction.WSS(symbol, [windField], params)
                print("Missing " + symbol + " " + rptDateStr + "  " + field + " FilledBy ")
                kkwood = 0


def Fix_ZeroField(database, field, windField):
    instruments = database.findWithFilter("Instruments", "Stock")
    count = 0
    start = False
    for instrument in instruments:
        count = count + 1
        symbol = instrument["Symbol"]

        #if symbol == "600005.SH":
        #    start = True
        #if start == False:
        #    continue

        print("Count " + symbol + " " + str(count))
        collectionName = symbol + "_Fundamental"
        fundamentals = database.findAll("Fundamental", collectionName)
        if len(fundamentals) == 0:
            # print(symbol + " NoReport")
            continue

        #
        for statement in fundamentals:
            if field in statement["Values"] and statement["Values"][field] == 0:
                #
                rptDateStr = statement["ReportDate"]
                rptDateStr = rptDateStr[:10]
                params = {}
                params["rptDate"] = rptDateStr
                # params["endDate"] = rptDateStr
                params["tradeDate"] = rptDateStr
                # params["currencyType"] = ""
                params["rptType"] = "1"
                # params["priceAdj"] = "U"
                # params["cycle"] = "D"
                #
                values = WindFunction.WSS(symbol, [windField], params)
                print("Zero " + symbol + " " + rptDateStr + "  " + field + " FilledBy ")
                kkwood = 0


# Fundamental 中的Close字段因为当时忽略了 TradeDate 字段导致有错误
# 在此修复
def Fix_CloseInFundamental(database, startIndex=0):
    #
    instruments = database.Find("financial_data", "instrument_stock")
    #
    i = 0
    for instrument in instruments:
        symbol = instrument["symbol"]
        i += 1
        if i < 77:
            continue
        print(i, symbol)
        dfBars = database.GetDataFrame2(symbol, dataType="dailybar", instrumentType="stock")

        #
        reports = database.Find("financial_data", "stock_fundamental", filter={"Symbol": symbol})
        for report in reports:
            #
            reportDate = report["report_date"]
            strReportDate = Gadget.ToDateString(reportDate)
            #
            dfTmp = dfBars[dfBars["Date"] <= reportDate]
            if len(dfTmp) == 0:
                continue
            #
            dfTmp = dfTmp.iloc[-1]
            tradeDate = dfTmp["DateTime"]
            close = dfTmp["Close"]

            # d = WindFunction.w.wss(symbol, "close", "tradeDate=" + strReportDate + ";priceAdj=U;cycle=D")
            print(symbol, reportDate, tradeDate, close)
            #
            target = {}
            target["Symbol"] = symbol
            target["Report_Date"] = reportDate
            #
            document = {"close": close}
            database.Update("financial_data", "stock_fundamental", target, document)
            database.Update("financial_data", "stock_fundamental_basic", target, document)
            a = 0
        #
        b = 0


def Fix_PreferredStock_CommonStock(database):

    count = database.Count("financial_data", "stock_fundamental_basic")
    # count = 8780000
    batchSize = 10000
    batchCount = int(count / batchSize) + 1

    filter = {}
    filter["limit"] = batchSize
    #
    for i in range(batchCount):
        print("inserting", i, batchCount, datetime.now())
        filter["skip"] = i * batchSize
        bars = database.Find("financial_data", "stock_fundamental_basic", filter)
        #
        newBars = []
        for bar in bars:
            target = {"Key2": bar["key2"]}
            document = {"preferredstock": bar["other_equity_instruments_pre"], "commonstock": bar["cap_stk"]}
            database.Update("financial_data", "stock_fundamental", target, document)
            #
            # newBars.append(newBar)
            a = 0
            pass

        # toMysql.Insert_Many(toDatabase, toTablename, newBars)
        # mysql.Upsert_Many("DailyBar", instrumentType, [], newBars)
        # toMysql.Upsert_Many(toDatabase, toTablename, [], newBars)


# --- 2018.4 ExpendFields ---
def ExpendFields(database,instruments):
    count = 0

    #
    for instrument in instruments:

        fields = []
        fields.append("investcapital")
        fields.append("interestdebt")
        fields.append("tangibleasset")
        fields.append("retainedearnings")
        fields.append("exinterestdebt_current")
        fields.append("exinterestdebt_noncurrent")
        #
        fields.append("r_and_d_costs")
        fields.append("deferred_inc_non_cur_liab")

        if instrument["Properties"]["Industry"] == "银行":
            fields.append("borrow_central_bank")


        count = count + 1
        symbol = instrument["Symbol"]
        print(symbol + " count:" + str(count))
        if count <=2050:
            continue
        collectionName = symbol + "_Fundamental"
        fundamentals = database.findAll("Fundamental",collectionName)
        if len(fundamentals) == 0:
            print(symbol + " NoReport")
            continue

        for statement in fundamentals:
            reportDate = statement["ReportDate"]
            print("ReportDate " + reportDate)
            reportDate = Gadget.ParseDateTime(reportDate)
            DataProcess.Download_Fundamentals_Fields(database, symbol, reportDate, fields)


# --- 2018.4.24 ExpendFields ---
def ExpendFields20180424(database,instruments):
    count = 0

    #
    for instrument in instruments:

        fields = []
        fields.append("other_grants_inc")
        fields.append("gain_asset_dispositions")

        count = count + 1
        symbol = instrument["Symbol"]
        print(symbol + " count:" + str(count))
        #if count <=2050:
        #    continue
        collectionName = symbol + "_Fundamental"
        fundamentals = database.findAll("Fundamental",collectionName)
        if len(fundamentals) == 0:
            print(symbol + " NoReport")
            continue

        for statement in fundamentals:
            reportDate = statement["ReportDate"]
            print("Expend Field @ ReportDate " + reportDate)
            reportDate = Gadget.ParseDateTime(reportDate)
            DataProcess.Download_Fundamentals_Fields(database, symbol, reportDate, fields)


# --- 2020-4-1 ExpendFields ---
# 主要针对银行
def ExpendFields20200401(database, startIndex=0):
    #
    instruments = database.Find("Financial_Data", "Instrument_Stock")
    #
    count = 0
    for instrument in instruments:
        #
        fields = []
        fields.append("hfs_assets")  # 划分为持有待售的资产
        fields.append("fin_assets_chg_compreh_inc")  # 以公允价值计量且其变动计入其他综合收益的金融资产
        fields.append("fin_assets_amortizedcost")  # 以摊余成本计量的金融资产
        fields.append("fin_invest")
        # fields.append("debt_invest")
        # fields.append("oth_debt_invest")
        # fields.append("oth_eqy_instruments_invest")

        count = count + 1
        if count < startIndex:
            continue

        #
        symbol = instrument["symbol"]
        print(symbol, " count:", str(count))

        #
        fundamentals = database.Find("Financial_Data", "Stock_Fundamental_Basic", filter={"Symbol": symbol})
        if len(fundamentals) == 0:
            print(symbol + " NoReport")
            continue

        for statement in fundamentals:
            reportDate = statement["report_date"]
            print("Expend Field @ ReportDate ", reportDate)
            DataProcess.Download_Fundamentals_Fields(database, symbol, reportDate, fields, isUpdate=True)

        # 顺便修正字段
        # fundamentals = database.Find("Financial_Data", "Stock_Fundamental_Basic", filter={"Symbol": symbol})
        # for statement in fundamentals:
        #     #
        #     for k, value in statement.items():
        #         if isinstance(value, decimal.Decimal):
        #             statement[k] = float(value)
        #         if value == None:
        #             statement[k] = 0
        #     #
        #     newValue = DataProcess.Calc_STInvest(statement)
        #     target = {"key2": statement["key2"]}
        #     document = {"STInvest": newValue}
        #     database.Update("financial_data", "stock_fundamental", target, document)
        #
        a = 0


def ExpendFields20200407(database, startIndex=0):
    #
    instruments = database.Find("Financial_Data", "Instrument_Stock")
    #
    count = 0
    for instrument in instruments:
        #
        fields = []
        fields.append("rd_exp")  # 研发费用
        count = count + 1
        if count < startIndex:
            continue
        #
        symbol = instrument["symbol"]
        print(symbol, " count:", str(count))
        #
        fundamentals = database.Find("Financial_Data", "Stock_Fundamental_Basic", filter={"Symbol": symbol})
        if len(fundamentals) == 0:
            print(symbol + " NoReport")
            continue
        #
        for statement in fundamentals:
            reportDate = statement["report_date"]
            print("Expend Field @ ReportDate ", reportDate)
            DataProcess.Download_Fundamentals_Fields(database, symbol, reportDate, fields, isUpdate=True)
        #
        a = 0



def ExpendFields20201127(database, startIndex=0):
    #
    instruments = database.Find("Financial_Data", "Instrument_Stock")
    #
    count = 0
    for instrument in instruments:
        #
        fields = []
        fields.append("fin_invest")  # 金融投资 = 交易性金融资产 + 其他权益工具投资
        fields.append("tradable_fin_assets")  # 交易性金融资产
        fields.append("oth_eqy_instruments_invest")  # 其他权益工具投资
        fields.append("derivative_fin_assets")  # 衍生金融资产
        #
        fields.append("hfs_assets")  # 划分为持有待售的资产
        fields.append("fin_assets_avail_for_sale")  # 可供出售金融资产
        fields.append("held_to_mty_invest")  # 持有至到期投资
        #
        fields.append("fin_assets_chg_compreh_inc")  # 以公允价值计量且其变动计入其他综合收益的金融资产
        fields.append("fin_assets_amortizedcost")  # 以摊余成本计量的金融资产

        count = count + 1
        if count < startIndex:
            continue
        #
        symbol = instrument["symbol"]
        print(symbol, " count:", str(count), datetime.now())

        #
        fundamentals = database.Find("Financial_Data", "Stock_Fundamental_Basic", filter={"Symbol": symbol})
        if len(fundamentals) == 0:
            print(symbol + " NoReport")
            continue

        for statement in fundamentals:
            reportDate = statement["report_date"]
            # print("Expend Field @ ReportDate ", reportDate)
            DataProcess.Download_Fundamentals_Fields(database, symbol, reportDate, fields, isUpdate=True)
        #
        a = 0


if __name__ == '__main__':
    #
    from Core.Config import *
    pathfilename = os.getcwd() + "\..\Config\config2.json"
    config = Config(pathfilename)
    database = config.DataBase("JDMySQL")
    logger = config.Logger("WindFundamental")

    #
    WindFunction.w.start() # 启动Wind API

    # ---扩展字段---
    # ExpendFields(database, instruments)
    # ExpendFields20180424(database, instruments)
    # ExpendFields20200401(database, 2773)
    # ExpendFields20200407(database, startIndex=1841)
    ExpendFields20201127(database, startIndex=2691)

    # ---补齐数据---
    datetime1 = datetime(2005, 1, 1)
    datetime2 = datetime(2019, 7, 1)
    # Download_Top10ShareHolders(database, datetime1, datetime2)
    # Download_MissingReport(database)
    # missing = MissingShareHolders20180930()
    # Download_MissingShareHolders(database, missing)
    # Fundamental.Download_Fundamental(database, "600005.SH", datetime(2016,12,31))
    # 000022.SZ
    # DataProcess.Batch_Download_Fundamentals(database, datetime1, datetime2, startIndex=0)

    # ---检查数据---
    instruments = []
    filter = {}
    # filter["Symbol"] = "000768.SZ"
    # ipoDate = datetime(2005,1,1)
    # filter["IPO_Date"] =  {"$gt": ipoDate}
    instruments = database.Find("Instruments", "Stock", filter)
    datetime1 = datetime(2005, 1, 1)
    datetime2 = datetime(2020, 1, 1)
    # missingReports = CheckMissingReport(database,instruments,datetime1,datetime2)
    # CheckDuplicated(database, instruments, datetime1, datetime2)
    # CheckField(database,field="tsy_stk")
    # CheckMissingWindValues(database, instruments)
    # CheckMissingShareHolders(database, instruments, datetime1, datetime2)
    k = 1

    # ---修正数据---
    # Fix_FreeCashFlow(database, [{"Symbol":"000333.SZ"}])
    # Fix_FreeCashFlow(database, [{"Symbol":"600823.SH"}])
    # Fundamental.FormatingReports(database, instruments)

    # Fix_OpratingProfit1(database,instruments)
    # Fix_Unset(database,instruments)
    # FormatingReport(database,instruments)

    # Fix_MissingField(database, "TotalEquity", "tot_equity")
    # Fix_ZeroField(database, "TotalEquity", "tot_equity")
    # Fix_ZeroField(database, "TotalShares", "total_shares")
    # Fix_CloseInFundamental(database)
    # Fix_PreferredStock_CommonStock(database)

    # ---精确修复---
    # symbol = "002821.SZ"
    # reportDate = datetime(2017,12,31)
    # Fundamental.Request_Write_Fundamentals_Dividend(database, symbol, reportDate)